'''
需要开启vpn或者部署在国外服务器
'''
# -*- coding: utf-8 -*-
import requests
from lxml import etree
import re
import traceback
import time
from loguru import logger
import random
import datetime
import execjs
import threading
import sys
import os
import json
from queue import Queue, Empty
from fake_useragent import UserAgent
import concurrent.futures
from src.service.requests_middler import requests_sys
from src.service.requests_middler import proxy_requests_sys
from src.service.requests_middler import insert_shipping_info
from src.service.requests_middler import read_excel
from src.service.requests_middler import read_transport_json
from src.service.requests_middler import get_proxies
current_working_directory = os.getcwd()
ip_queue = Queue()
ua = UserAgent()
random_user_agent = ua.random
stop_threading_event = threading.Event()

total_requests = 0
loss_requests = 0

def hmm21_transport(origin_city, destination_city):
    global total_requests
    global loss_requests
    port_mapping = {}
    # file_path = "../maeu_ports.json"
    file_path = r'/src/crawler' + "/maeu_ports.json" # linux
    # file_path = current_working_directory + file_path
    # file_path = os.path.join(os.path.dirname(os.getcwd()), 'maeu_ports.json') # windwos
    file_path = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), 'maeu_ports.json')
    if os.path.exists(file_path):
        with open(file_path, "r") as file:
            Menu_json = json.load(file)
    else:
        logger.error(f"不存在文件: {file_path}")
        sys.exit()
    for info in Menu_json['ports']:
        port_mapping[info['portName'].upper()] = {
            'portCode': info.get('portCode'),
            'unLocationCode': info['unLocationCode'],
            "regionCode": info.get('regionCode')
        }
    origin_info = port_mapping.get(origin_city.upper())
    destination_info = port_mapping.get(destination_city.upper())
    if origin_info and destination_info:
        formatted_date = datetime.datetime.now().strftime("%Y%m%d")
        origin_unLocationCode = origin_info['unLocationCode']
        des_unLocationCode = destination_info['unLocationCode']
        regionCode = destination_info.get('regionCode', None)
        total_requests += 1
        retry_count = 0
        if origin_city == 'NINGBO':
            origin_unLocationCode = 'CNNGB'
        elif origin_city.upper() == 'TIANJIN':
            origin_unLocationCode = 'CNTSN'
        print(origin_unLocationCode)
        while retry_count < 3:
            headers = {
                "accept": "*/*",
                "accept-language": "zh-CN,zh;q=0.9",
                "cache-control": "no-cache",
                "content-type": "application/json; charset=UTF-8",
                "origin": "https://www.hmm21.com",
                "pragma": "no-cache",
                "priority": "u=1, i",
                "referer": "https://www.hmm21.com/e-service/general/schedule/ScheduleMainPost.do",
                "sec-ch-ua": "\"Google Chrome\";v=\"129\", \"Not=A?Brand\";v=\"8\", \"Chromium\";v=\"129\"",
                "sec-ch-ua-mobile": "?0",
                "sec-ch-ua-platform": "\"Windows\"",
                "sec-fetch-dest": "empty",
                "sec-fetch-mode": "cors",
                "sec-fetch-site": "same-origin",
                "user-agent": random_user_agent,
                "x-csrf-token": "98e89298-5505-4e7f-a6e0-5b5f704af0d9",
                "x-requested-with": "XMLHttpRequest"
            }
            url = "https://www.hmm21.com/e-service/general/schedule/apiPointToPointList.do"
            data = {
                "srchViewType": "L",
                "srchPointFromCd": origin_unLocationCode,
                "srchCityFrom": "CY",
                "srchPointToCd": des_unLocationCode,
                "srchCityTo": "CY",
                # "srchCityTo": "DR",
                "srchSailDate": f"{formatted_date}",
                # "srchSailDate": f"{formatted_date}12",
                "srchSelWeeks": "4",
                "srchSelPriority": "A",
                "srchSelSortBy": "D",
                "srchPorFcltyCd": "",
                "srchPvyFcltyCd": "",
                "paramToday": f"{formatted_date}",
                # "paramToday": f"{formatted_date}12"
            }
            data = json.dumps(data, separators=(',', ':'))
            res = requests_sys(url, headers=headers, data=data, POST=True)
            if res:
                js_data = res.json()
                logger.info(f"当前查询 起点:{origin_city} ----> 终点: {destination_city}")
                logger.info(f"当前请求总数: {total_requests}, 失败请求总数: {loss_requests}")
                if js_data['RTN_STS'] == "OK":
                    GrmNo = js_data["RTN_DATA"]["resultData"]["GrmNo"]
                    logger.info(f"获取编号成功: {GrmNo}")
                    url = "https://www.hmm21.com/e-service/general/schedule/selectPointToPointList.do"
                    data = {
                        "srchViewType": "L",
                        "srchGrmNo": GrmNo,
                        "srchSelPriority": "A",
                        "srchSelSortBy": "D",
                        "isNew": True
                    }
                    #O241018085777345 O241018085773538
                    data = json.dumps(data, separators=(',', ':'))
                    res = requests_sys(url, headers=headers, data=data, POST=True)
                    if res:
                        js_data = res.json()
                        if js_data.get("grmData"):
                            for info in js_data['grmData']:
                                item = {}
                                item['origin_code'] = f"{origin_city}({origin_info['unLocationCode']})"
                                if regionCode:
                                    item['destination_code'] = f"{destination_city}, {regionCode}({des_unLocationCode})"
                                else:
                                    item['destination_code'] = f"{destination_city}({des_unLocationCode})"
                                des_city = info['transit'][-1]['destnLocNm'].split(',')[0]
                                item['depart_date'] = datetime.datetime.strptime(info["transit"][0]["arvlStDt"], "%Y-%m-%dT%H:%M:%S").strftime("%Y-%m-%d")
                                item['arrive_date'] = datetime.datetime.strptime(info["transit"][-1]["dpartFnshDt"], "%Y-%m-%dT%H:%M:%S").strftime("%Y-%m-%d")
                                item['transport_contractor'] = "HMM"
                                date1 = datetime.datetime.fromisoformat(info["transit"][0]["arvlStDt"])
                                date2 = datetime.datetime.fromisoformat(info["transit"][-1]["dpartFnshDt"])
                                date_difference = date2 - date1
                                item['haulage_time'] = date_difference.days
                                item['ship_name'] = info["transit"][0]["vslNm"] or info["transit"][1]["vslNm"] or info["transit"][-1].get("vslNm")
                                item['voyage_number'] = info["transit"][0]["cssmObVoyNo"] or info["transit"][1]["cssmObVoyNo"] or info["transit"][-1].get("cssmObVoyNo")
                                if info['transit'][-1]['trsPtnModeNm'] != 'VESSEL':
                                    # if len(info['transit']) == 2:
                                    #     item['transfer_code'] = f"{info['transit'][0]['destnLocNm'].split(',')[0]}({info['transit'][0]['destnLocCd']})"
                                    if len(info['transit']) >= 2:
                                        index = 1
                                        while True:
                                            trsPtnModeNm = info['transit'][-index]['trsPtnModeNm']
                                            if trsPtnModeNm == 'VESSEL':
                                                break
                                            index += 1
                                        #if info['transit'][-index]['destnLocCd'][0:2] == 'US' or info['transit'][-index]['destnLocCd'][0:2] == 'CA':
                                        item[
                                            'transfer_code'] = f"{info['transit'][-index]['destnLocNm'].split(',')[0]}({info['transit'][-index]['destnLocCd']})"
                                if item.get('transfer_code'):
                                    if not port_mapping.get(item['transfer_code'].split('(')[0].upper()):
                                        del item['transfer_code']
                                item['shipping_lanes'] = info["transit"][0]['vslSvcLoopCd']
                                print(item)
                                # insert_shipping_info(item)
                        else:

                            logger.info(f"起点:{origin_city} ----> 终点: {destination_city} 没有数据")
                            read_transport_json(transport='HMM', origin_city=origin_city, des_city=destination_city,
                                           fail_result="没有数据")
                            loss_requests += 1
                            retry_count += 1
                        break
                    elif res == 0:
                        logger.info(f"起点:{origin_city} ----> 终点: {destination_city} 请求超时")
                        retry_count += 1
                else:
                    logger.info(f"起点:{origin_city} ----> 终点: {destination_city} 查询编号失败")
                    read_transport_json(transport='HMM', origin_city=origin_city, des_city=destination_city,
                                   fail_result="查询编号失败")
                    loss_requests += 1
                    retry_count += 1
            elif res == 0:
                logger.info(f"起点:{origin_city} ----> 终点: {destination_city} 查询编号失败, 请求超时")
                read_transport_json(transport='HMM', origin_city=origin_city, des_city=destination_city,
                               fail_result="查询编号失败, 请求超时")
                retry_count += 1
        if retry_count == 3:
            read_transport_json(transport='HMM', origin_city=origin_city, des_city=destination_city,
                                fail_result="请求超时")
    else:
        logger.info(f"起点:{origin_city} ----> 终点: {destination_city} 查询失败")

#HMM
def process_route(origin_city, destination_list):
    for destination_city in destination_list:
        hmm21_transport(origin_city, destination_city)

def hmm21_spider_run():
    global total_requests
    global loss_requests
    # origin_city_list = [
    #     "Shanghai", "XIAMEN", "NANSHA", "QINGDAO", "YANTIAN", "NINGBO"
    # ]
    #
    # destination_list = [
    #     "Los Angeles", "Long Beach", "Atlanta", "Baltimore", "Boston", "Buffalo", "Calgary", "Charleston",
    #     "Chicago", "Chippewa Falls", "Cincinnati", "Cleveland", "Dallas", "Denver",
    #     "Detroit", "Edmonton", "EL Paso", "Halifax", "Houston", "Jacksonville",
    #     "Kansas City", "Louisville", "Memphis", "Miami", "Minneapolis", "Montreal",
    #     "Nashville", "New Orleans", "New York", "Norfolk", "Oakland", "Omaha",
    #     "Philadelphia", "Pittsburgh", "Portland", "Prince Rupert", "Saint Louis",
    #     "Salt Lake City", "San Antonio", "Saskatoon", "Savannah", "Seattle",
    #     "Tacoma", "Tampa", "Toronto", "Vancouver", "Winnipeg"
    # ]
    json_path_one = r"origin_and_destination.json"
    print(os.path.abspath(__file__))
    json_path = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), json_path_one)

    with open(f'{json_path}', 'r', encoding='utf-8') as f:
        tmp = json.loads(f.read())
    origin_city_list = tmp['origin_city_list']
    destination_list = tmp['destination_list']
    del tmp, json_path, json_path_one

    # with concurrent.futures.ThreadPoolExecutor() as executor:
    #     futures = [executor.submit(process_route, origin_city, destination_list) for origin_city in origin_city_list]

    with concurrent.futures.ThreadPoolExecutor() as executor:
        futures = []
        for origin_city in origin_city_list:
            future = executor.submit(process_route, origin_city, destination_list)
            futures.append(future)
            time.sleep(1)

        for future in concurrent.futures.as_completed(futures):
            try:
                future.result()
            except Exception as e:
                # 输出异常详细信息，包括出错的行号
                print(f"An error occurred: {e}")
                print(traceback.format_exc())
    read_excel(transport="HMM", total_requests=total_requests, loss_requests=loss_requests)

if __name__ == '__main__':
    hmm21_spider_run()